#!/usr/bin/perl -w
use warnings;
use strict;
use DBI;
require('config.pl');
require('function.pl');

my $db_conn = DBI->connect("DBI:mysql:database=58_spider;host=$ENV{'db_host'}", $ENV{'db_user'}, $ENV{'db_password'});
$db_conn->do('set names utf8');
my %cities = load_map('city_map_58');
my %urls = load_map('58_url');
load_proxies();
while (my ($city_chn, $city_en) = each %cities) {
	while (my ($url, $keyword) = each %urls) {
		$url =~ s/city/$city_en/;
		my $sum = get_ascii_sum($url);
		next if ($sum % $ARGV[0] != $ARGV[1]);
		print "$url\n";
		my $list_html = fetch_url_via_proxy($url);
		my @detail_urls = parse_list($list_html);
		foreach my $detail_url (@detail_urls) {
			next if (execute_scalar("select count(*) from spider_data where url = '$detail_url'", $db_conn) > 0);
			print "\t$detail_url\n";
			my $detail_html = fetch_url_via_proxy($detail_url);
			my ($title, $contact, $phone, $pub_date) = parse_detail($detail_html);
			print "$title\t$contact\t$phone\t$pub_date\n";
			next if (!defined($title) || !defined($contact) || !defined($phone) || !defined($pub_date));
			$title = $db_conn->quote($title);
			$contact = $db_conn->quote($contact);
			$db_conn->do("insert into spider_data(url, title, fetch_time, keyword1, keyword2, keyword3, keyword4) values('$detail_url', $title, '$pub_date', '$keyword', '$city_chn', $contact, '$phone')");
		}
	}
}

sub get_ascii_sum {
	my $sum = 0;
	for (my $i = 0; $i < length($_[0]); ++$i) {
		$sum += ord(substr($_[0], $i, 1));
	}
	return $sum;
}

sub parse_list {
	my @urls;
	while ($_[0] =~ /(http:\/\/[\w\/\.]+\.shtml)/g) {
		push @urls, $1;
	}
	return @urls;
}

sub parse_detail {
	my ($title, $contact, $phone, $pub_date);
	$title = $1 if ($_[0] =~ /<h1>([\d\D]+)<\/h1>/);
	if (!defined($title)) {
		$title = $1 if ($_[0] =~ /<h1.*?>([\d\D]+)<\/h1>/);
	}
	if (defined($title)) {
		$title =~ s/\s+//g;
	}
	$contact = $1 if ($_[0] =~ /姓名：<\/i>([\d\D]+?)<\/li>/);
	if (!defined($contact)) {
		$contact = $1 if ($_[0] =~ /linkman:'([\d\D]+?)'/);
	}
	if (!defined($contact)) {
		$contact = '没有';
	}
	$phone = $1 if ($_[0] =~ /(http:\/\/image.58.com\/showphone.aspx\?t=v55&v=\w+)/);
	$pub_date = $1 if ($_[0] =~ /发布时间：([\d\-]+)/);
	if (!defined($pub_date)) {
		$pub_date = $1 if ($_[0] =~ /<li class="time">([\d\.\-]+)<\/li>/);
		$pub_date =~ s/\./\-/g if (defined $pub_date);
	}
	if (!defined($pub_date)) {
		$pub_date = get_date_str(0);
	}
	print "parse failed\t$title\t$contact\t$phone\t$pub_date\n" if (!defined($title) || !defined($contact) || !defined($phone) || !defined($pub_date));
	return ($title, $contact, $phone, $pub_date);
}
